{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-17T04:16:30.078997Z",
     "start_time": "2020-09-17T04:16:30.077537Z"
    }
   },
   "source": [
    "# 导入工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T03:55:55.217270Z",
     "start_time": "2020-10-08T03:55:53.107402Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "from numpy.random import random\n",
    "from collections import defaultdict\n",
    "import json\n",
    "#稀疏矩阵，存储打分表\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "\n",
    "#数据到文件存储\n",
    "import pickle as pk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T03:59:28.587960Z",
     "start_time": "2020-10-08T03:59:28.488555Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>874965758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>876893171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>878542960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>876893119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>889751712</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  rating  timestamp\n",
       "0        1        1       5  874965758\n",
       "1        1        2       3  876893171\n",
       "2        1        3       4  878542960\n",
       "3        1        4       3  876893119\n",
       "4        1        5       3  889751712"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "triplet_cols = ['user_id', 'item_id', 'rating', 'timestamp']\n",
    "df_triplet = pd.read_csv(\"u1.base\", sep = '\\t', names = triplet_cols, encoding = 'latin - 1')\n",
    "df_triplet.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 建立user和item索引"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T07:17:10.061883Z",
     "start_time": "2020-10-08T07:17:10.049859Z"
    }
   },
   "outputs": [],
   "source": [
    "#统计user和item数目\n",
    "unique_users = df_triplet['user_id'].unique()\n",
    "unique_items = df_triplet['item_id'].unique()\n",
    "\n",
    "n_users = unique_users.shape[0]\n",
    "n_items = unique_items.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T07:18:11.655162Z",
     "start_time": "2020-10-08T07:18:11.650374Z"
    }
   },
   "outputs": [],
   "source": [
    "#建立用户和item的索引\n",
    "#本数据集中user_id and item_id都已经是索引了，可以减1，将从1开始编码变为从0开始编码\n",
    "users_index = dict()\n",
    "items_index = dict()\n",
    "\n",
    "for j, u in enumerate(unique_users):\n",
    "    users_index[u] = j \n",
    "    \n",
    "#重新编码活动索引字典     \n",
    "for j, i in enumerate(unique_items):\n",
    "    items_index[i] = j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T07:22:38.147481Z",
     "start_time": "2020-10-08T07:22:22.130747Z"
    }
   },
   "outputs": [],
   "source": [
    "#建立倒排表（统计每个用户打过分的电影，以及每个电影被哪些用户打过分）\n",
    "user_items = defaultdict(set)\n",
    "item_users = defaultdict(set)\n",
    "\n",
    "#用户-item关系矩阵R,稀疏矩阵，记录每个用户对电影的打分\n",
    "user_item_scores = ss.dok_matrix((n_users, n_items))\n",
    "\n",
    "#扫描训练数据\n",
    "for line in range(0, len(df_triplet.index)):#对每条记录\n",
    "    cur_user_index = users_index [df_triplet.iloc[line]['user_id']]\n",
    "    cur_item_index = items_index [df_triplet.iloc[line]['item_id']]\n",
    "    \n",
    "    #到排序\n",
    "    user_items[cur_user_index].add(cur_item_index)#该用户对这个电影进行了打分\n",
    "    item_users[cur_item_index].add(cur_user_index)#该电影被改用户打分\n",
    "    \n",
    "    user_item_scores[cur_user_index, cur_item_index] = df_triplet.iloc[line]['rating']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 初始化模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T07:33:22.605324Z",
     "start_time": "2020-10-08T07:33:22.579477Z"
    }
   },
   "outputs": [],
   "source": [
    "# 隐含变量的维数\n",
    "K = 40\n",
    "\n",
    "# r(ui) = \\mu + bu + bi + Pu'T Qi\n",
    "\n",
    "# Item和User的偏置项\n",
    "bi = np.zeros((n_items, 1))\n",
    "bu = np.zeros((n_users, 1))\n",
    "\n",
    "# Item和User的隐含向量\n",
    "qi = np.zeros((n_items, K))    \n",
    "pu = np.zeros((n_users, K))   \n",
    "\n",
    "for uid in range(n_users):\n",
    "    pu[uid] = np.reshape(random((K, 1)) / 10 * np.sqrt(K), K)\n",
    "       \n",
    "for iid in range(n_items):\n",
    "    qi[iid] = np.reshape(random((K, 1)) / 10 * np.sqrt(K), K)\n",
    "\n",
    "# 所有用户的平均打分\n",
    "mu = df_triplet['rating'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 根据当前参数，预测用户uid对Item（i_id）的打分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T07:33:33.753825Z",
     "start_time": "2020-10-08T07:33:33.749835Z"
    }
   },
   "outputs": [],
   "source": [
    "def svd_pred(uid, iid):  \n",
    "    score = mu + bi[iid] + bu[uid] + np.sum(qi[iid] * pu[uid]) \n",
    "    return score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T07:49:15.029801Z",
     "start_time": "2020-10-08T07:34:35.549463Z"
    },
    "lang": "en"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "开始进行50个step的训练\n",
      "The 0-th  step is running\n",
      "完成第1个step的训练, rmse=[ 1.18477735], 耗时17.5060秒\n",
      "The 1-th  step is running\n",
      "完成第2个step的训练, rmse=[ 0.92673679], 耗时17.6478秒\n",
      "The 2-th  step is running\n",
      "完成第3个step的训练, rmse=[ 0.90825251], 耗时19.6684秒\n",
      "The 3-th  step is running\n",
      "完成第4个step的训练, rmse=[ 0.89688501], 耗时18.2911秒\n",
      "The 4-th  step is running\n",
      "完成第5个step的训练, rmse=[ 0.88799148], 耗时18.5474秒\n",
      "The 5-th  step is running\n",
      "完成第6个step的训练, rmse=[ 0.87977084], 耗时18.8147秒\n",
      "The 6-th  step is running\n",
      "完成第7个step的训练, rmse=[ 0.87206992], 耗时18.3649秒\n",
      "The 7-th  step is running\n",
      "完成第8个step的训练, rmse=[ 0.86558835], 耗时17.3517秒\n",
      "The 8-th  step is running\n",
      "完成第9个step的训练, rmse=[ 0.85952522], 耗时17.6967秒\n",
      "The 9-th  step is running\n",
      "完成第10个step的训练, rmse=[ 0.85467858], 耗时18.1655秒\n",
      "The 10-th  step is running\n",
      "完成第11个step的训练, rmse=[ 0.84996816], 耗时18.6173秒\n",
      "The 11-th  step is running\n",
      "完成第12个step的训练, rmse=[ 0.84617206], 耗时18.1375秒\n",
      "The 12-th  step is running\n",
      "完成第13个step的训练, rmse=[ 0.84300788], 耗时17.4474秒\n",
      "The 13-th  step is running\n",
      "完成第14个step的训练, rmse=[ 0.84013201], 耗时17.0036秒\n",
      "The 14-th  step is running\n",
      "完成第15个step的训练, rmse=[ 0.83707334], 耗时17.1462秒\n",
      "The 15-th  step is running\n",
      "完成第16个step的训练, rmse=[ 0.83476424], 耗时16.9527秒\n",
      "The 16-th  step is running\n",
      "完成第17个step的训练, rmse=[ 0.83277816], 耗时17.3695秒\n",
      "The 17-th  step is running\n",
      "完成第18个step的训练, rmse=[ 0.83062159], 耗时17.0176秒\n",
      "The 18-th  step is running\n",
      "完成第19个step的训练, rmse=[ 0.82868613], 耗时17.5550秒\n",
      "The 19-th  step is running\n",
      "完成第20个step的训练, rmse=[ 0.8272498], 耗时17.0824秒\n",
      "The 20-th  step is running\n",
      "完成第21个step的训练, rmse=[ 0.82569395], 耗时17.3247秒\n",
      "The 21-th  step is running\n",
      "完成第22个step的训练, rmse=[ 0.82418475], 耗时17.9341秒\n",
      "The 22-th  step is running\n",
      "完成第23个step的训练, rmse=[ 0.82312497], 耗时17.7555秒\n",
      "The 23-th  step is running\n",
      "完成第24个step的训练, rmse=[ 0.82192061], 耗时17.1901秒\n",
      "The 24-th  step is running\n",
      "完成第25个step的训练, rmse=[ 0.82081318], 耗时16.9208秒\n",
      "The 25-th  step is running\n",
      "完成第26个step的训练, rmse=[ 0.81987045], 耗时17.1242秒\n",
      "The 26-th  step is running\n",
      "完成第27个step的训练, rmse=[ 0.81890769], 耗时17.7506秒\n",
      "The 27-th  step is running\n",
      "完成第28个step的训练, rmse=[ 0.81806552], 耗时17.1551秒\n",
      "The 28-th  step is running\n",
      "完成第29个step的训练, rmse=[ 0.81720903], 耗时17.6997秒\n",
      "The 29-th  step is running\n",
      "完成第30个step的训练, rmse=[ 0.81658349], 耗时19.2545秒\n",
      "The 30-th  step is running\n",
      "完成第31个step的训练, rmse=[ 0.81593871], 耗时17.7705秒\n",
      "The 31-th  step is running\n",
      "完成第32个step的训练, rmse=[ 0.81528001], 耗时17.0924秒\n",
      "The 32-th  step is running\n",
      "完成第33个step的训练, rmse=[ 0.81479436], 耗时17.1163秒\n",
      "The 33-th  step is running\n",
      "完成第34个step的训练, rmse=[ 0.8141794], 耗时17.2678秒\n",
      "The 34-th  step is running\n",
      "完成第35个step的训练, rmse=[ 0.81380169], 耗时18.4806秒\n",
      "The 35-th  step is running\n",
      "完成第36个step的训练, rmse=[ 0.81332277], 耗时17.6029秒\n",
      "The 36-th  step is running\n",
      "完成第37个step的训练, rmse=[ 0.81284723], 耗时17.5870秒\n",
      "The 37-th  step is running\n",
      "完成第38个step的训练, rmse=[ 0.81251236], 耗时17.4305秒\n",
      "The 38-th  step is running\n",
      "完成第39个step的训练, rmse=[ 0.81218163], 耗时17.6259秒\n",
      "The 39-th  step is running\n",
      "完成第40个step的训练, rmse=[ 0.8118565], 耗时17.2997秒\n",
      "The 40-th  step is running\n",
      "完成第41个step的训练, rmse=[ 0.81153961], 耗时17.0414秒\n",
      "The 41-th  step is running\n",
      "完成第42个step的训练, rmse=[ 0.81116164], 耗时17.1741秒\n",
      "The 42-th  step is running\n",
      "完成第43个step的训练, rmse=[ 0.81092735], 耗时17.2908秒\n",
      "The 43-th  step is running\n",
      "完成第44个step的训练, rmse=[ 0.81066617], 耗时17.3187秒\n",
      "The 44-th  step is running\n",
      "完成第45个step的训练, rmse=[ 0.81046798], 耗时17.2330秒\n",
      "The 45-th  step is running\n",
      "完成第46个step的训练, rmse=[ 0.81025871], 耗时17.2469秒\n",
      "The 46-th  step is running\n",
      "完成第47个step的训练, rmse=[ 0.81005686], 耗时16.9986秒\n",
      "The 47-th  step is running\n",
      "完成第48个step的训练, rmse=[ 0.80985268], 耗时17.4374秒\n",
      "The 48-th  step is running\n",
      "完成第49个step的训练, rmse=[ 0.80967718], 耗时16.9198秒\n",
      "The 49-th  step is running\n",
      "完成第50个step的训练, rmse=[ 0.80953578], 耗时17.0464秒\n",
      "结束了50个step的训练，总耗时879.4734秒\n"
     ]
    }
   ],
   "source": [
    "# gamma：为学习率\n",
    "# Lambda：正则参数\n",
    "# steps：迭代次数\n",
    "import time\n",
    "\n",
    "steps = 50\n",
    "gamma = 0.04\n",
    "Lambda = 0.15\n",
    "\n",
    "# 总的打分记录数目\n",
    "n_records = df_triplet.shape[0]\n",
    "\n",
    "time_start = time.time()\n",
    "\n",
    "print(\"开始进行{}个step的训练\".format(steps))\n",
    "\n",
    "each_time_start = time_start\n",
    "for step in range(steps):\n",
    "    print('The {}-th  step is running'.format(step))\n",
    "    rmse_sum = 0.0\n",
    "            \n",
    "    # 将训练样本打散顺序\n",
    "    kk = np.random.permutation(n_records)  \n",
    "    for j in range(n_records):\n",
    "        # 每次一个训练样本\n",
    "        line = kk[j]\n",
    "        \n",
    "        uid = users_index[df_triplet.iloc[line]['user_id']]\n",
    "        iid = items_index[df_triplet.iloc[line]['item_id']]\n",
    "        rating = df_triplet.iloc[line]['rating']\n",
    "\n",
    "        # 预测残差\n",
    "        eui = rating - svd_pred(uid, iid)\n",
    "        # 残差平方和\n",
    "        rmse_sum += eui**2\n",
    "\n",
    "        # 随机梯度下降，更新\n",
    "        bu[uid] += gamma * (eui - Lambda * bu[uid])  \n",
    "        bi[iid] += gamma * (eui - Lambda * bi[iid])\n",
    "                \n",
    "        temp = qi[iid]  \n",
    "        qi[iid] += gamma * (eui * pu[uid] - Lambda * qi[iid])\n",
    "        pu[uid] += gamma * (eui * temp - Lambda * pu[uid])\n",
    "            \n",
    "    # 学习率递减\n",
    "    gamma = gamma * 0.93\n",
    "    each_rmse = np.sqrt(rmse_sum / n_records)\n",
    "    \n",
    "    each_time_tick = time.time()\n",
    "    each_cost_time = each_time_tick - each_time_start\n",
    "    each_time_start = each_time_tick\n",
    "\n",
    "    print(\"完成第{}个step的训练, rmse={}, 耗时{:.4f}秒\".format(\n",
    "        step + 1, each_rmse, each_cost_time))\n",
    "\n",
    "time_end = time.time()\n",
    "total_cost_time = time_end - time_start\n",
    "print(\"结束了{}个step的训练，总耗时{:.4f}秒\".format(steps, total_cost_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 保存模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T07:57:20.236769Z",
     "start_time": "2020-10-08T07:57:20.233804Z"
    }
   },
   "outputs": [],
   "source": [
    "# A method for saving object data to JSON file\n",
    "def save_json(filepath):\n",
    "    dict_ = dict()\n",
    "    dict_['mu'] = mu\n",
    "    dict_['K'] = K\n",
    "    \n",
    "    dict_['bi'] = bi.tolist()\n",
    "    dict_['bu'] = bu.tolist()\n",
    "    \n",
    "    dict_['qi'] = qi.tolist()\n",
    "    dict_['pu'] = pu.tolist()\n",
    "\n",
    "    # Creat json and save to file\n",
    "    json_txt = json.dumps(dict_)\n",
    "    with open(filepath, 'w') as file:\n",
    "        file.write(json_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T07:57:35.452940Z",
     "start_time": "2020-10-08T07:57:35.448910Z"
    }
   },
   "outputs": [],
   "source": [
    "# A method for loading data from JSON file\n",
    "def load_json(filepath):\n",
    "    with open(filepath, 'r') as file:\n",
    "        dict_ = json.load(file)\n",
    "\n",
    "        mu = dict_['mu']\n",
    "        K = dict_['K']\n",
    "\n",
    "        bi = np.asarray(dict_['bi'])\n",
    "        bu = np.asarray(dict_['bu'])\n",
    "    \n",
    "        qi = np.asarray(dict_['qi'])\n",
    "        pu = np.asarray(dict_['pu'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T07:58:50.011636Z",
     "start_time": "2020-10-08T07:58:49.849790Z"
    }
   },
   "outputs": [],
   "source": [
    "save_json('svd_model.json')\n",
    "load_json('svd_model.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 测试"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 对给定用户，推荐物品/计算打分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T07:59:34.241868Z",
     "start_time": "2020-10-08T07:59:34.235883Z"
    }
   },
   "outputs": [],
   "source": [
    "def recommend(user):\n",
    "    \"\"\"\n",
    "    返回推荐items及其打分（DataFrame）\n",
    "    :param user: 输出User的userId\n",
    "    :return: 推荐结果DataFrame, 包含item_id和score\n",
    "    \"\"\"\n",
    "    cur_user_id = users_index[user]\n",
    "    \n",
    "    # 训练集中该用户打过分的item\n",
    "    cur_user_items = user_items[cur_user_id]\n",
    "\n",
    "    # 该用户对所有item的打分\n",
    "    user_items_scores = np.zeros(n_items)\n",
    "\n",
    "    # 预测打分\n",
    "    for i in range(n_items):\n",
    "        if i not in cur_user_items:\n",
    "            user_items_scores[i] = svd_pred(cur_user_id, i)\n",
    "    \n",
    "    # 推荐\n",
    "    sort_list = [(e, i) for i, e in enumerate(list(user_items_scores))]\n",
    "    sort_index = sorted(sort_list, reverse=True)\n",
    "    \n",
    "    # 创建DataFrame, 返回推荐列表\n",
    "    columns = ['item_id', 'score']\n",
    "    df = pd.DataFrame(columns=columns)\n",
    "\n",
    "    for each_item_score, each_item_index in sort_index:\n",
    "        if np.isnan(each_item_score):\n",
    "            continue\n",
    "        \n",
    "        if each_item_index in cur_user_items:\n",
    "            continue\n",
    "        \n",
    "        each_item_key_index = list(items_index.values()).index(each_item_index)\n",
    "        each_item_id = list(items_index.keys())[each_item_key_index]\n",
    "        df.loc[len(df)] = [each_item_id, each_item_score]\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取测试数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T08:00:28.713145Z",
     "start_time": "2020-10-08T08:00:28.639342Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>887431973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>875693118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>878542960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>5</td>\n",
       "      <td>874965706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>875073198</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  rating  timestamp\n",
       "0        1        6       5  887431973\n",
       "1        1       10       3  875693118\n",
       "2        1       12       5  878542960\n",
       "3        1       14       5  874965706\n",
       "4        1       17       3  875073198"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取测试数据\n",
    "triplet_cols = ['user_id', 'item_id', 'rating', 'timestamp'] \n",
    "df_triplet_test = pd.read_csv(\"u1.test\",sep='\\t', names=triplet_cols, encoding='latin-1')\n",
    "df_triplet_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试，并计算评价指标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T08:06:50.046302Z",
     "start_time": "2020-10-08T08:06:50.041311Z"
    }
   },
   "outputs": [],
   "source": [
    "def recommend_users(user_ids):\n",
    "    \"\"\"\n",
    "    为指定一批User推荐Item, 并返回推荐结果\n",
    "    :param user_ids: userId列表\n",
    "    :return: \n",
    "    \"\"\"\n",
    "    userid_rec_results = dict()\n",
    "    \n",
    "    time_start = time.time()\n",
    "    each_time_start = time_start\n",
    "    count = 0\n",
    "    \n",
    "    print(\"开始对{}个User计算推荐结果\".format(len(user_ids)))\n",
    "    for user_id in user_ids:\n",
    "        count = count + 1\n",
    "        if user_id not in users_index:\n",
    "            print(\"{} is a new user.\".format(user_id))\n",
    "            continue\n",
    "        \n",
    "        each_rec_items = recommend(user_id)\n",
    "        userid_rec_results[user_id] = each_rec_items\n",
    "        \n",
    "        if count % 100 == 0:\n",
    "            each_time_tick = time.time()\n",
    "            each_cost_time = each_time_tick - each_time_start\n",
    "            each_time_start = each_time_tick\n",
    "            print(\"user_index:{}, each_cost_time:{}\".format(\n",
    "                    count - 1, each_cost_time))\n",
    "            \n",
    "    time_end = time.time()\n",
    "    total_cost_time = time_end - time_start\n",
    "    print(\"完成{}个User的推荐, 共计耗时{}秒\".format(\n",
    "        count, total_cost_time\n",
    "    ))\n",
    "\n",
    "    return userid_rec_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T08:15:17.211984Z",
     "start_time": "2020-10-08T08:07:02.661091Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "开始对459个User计算推荐结果\n",
      "user_index:99, each_cost_time:108.65761137008667\n",
      "user_index:199, each_cost_time:109.69190907478333\n",
      "user_index:299, each_cost_time:107.92458629608154\n",
      "user_index:399, each_cost_time:106.69109344482422\n",
      "完成459个User的推荐, 共计耗时494.5466454029083秒\n"
     ]
    }
   ],
   "source": [
    "# 测试集中的user_id集合\n",
    "unique_users_test = df_triplet_test['user_id'].unique()\n",
    "total_rec_results = recommend_users(unique_users_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T08:21:13.045177Z",
     "start_time": "2020-10-08T08:21:13.039193Z"
    }
   },
   "outputs": [],
   "source": [
    "def evaluate(userid_rec_results, n_rec_items):\n",
    "    \"\"\"\n",
    "    为每个User截取若干条推荐结果，并衡量全体User的推荐效果\n",
    "    :param userid_rec_results:  推荐结果<userId -- 该user被推荐的全部Item>\n",
    "    :param n_rec_items:         每个所需截取的推荐结果的数量\n",
    "    :return: \n",
    "    \"\"\"\n",
    "    n_hits = 0              # TP\n",
    "    n_total_rec_items = 0   # TP+FP\n",
    "    n_total_test_items = 0  # TP+FN\n",
    "    all_rec_items = set()   # 所有User被推荐的Item的汇总\n",
    "    rss_test = 0.0          # 所有User的预测残差平方和\n",
    "\n",
    "    for user_id, rec_items in userid_rec_results.items():\n",
    "        user_records_test = df_triplet_test[df_triplet_test.user_id == user_id]\n",
    "        \n",
    "        # 累积TP和汇总所有User被推荐的指定数量的Item\n",
    "        for i in range(n_rec_items):\n",
    "            item_id = rec_items.iloc[i]['item_id']\n",
    "            if item_id in user_records_test['item_id'].values:\n",
    "                n_hits = n_hits + 1\n",
    "            all_rec_items.add(item_id)\n",
    "\n",
    "        # 累积预测评分残差平方和\n",
    "        for i in range(user_records_test.shape[0]):\n",
    "            item = user_records_test.iloc[i]['item_id']\n",
    "            score = user_records_test.iloc[i]['rating']\n",
    "            \n",
    "            df1 = rec_items[rec_items.item_id == item]\n",
    "            if df1.shape[0] == 0:\n",
    "                print(\"{} is a new item.\".format(item))\n",
    "                continue\n",
    "            \n",
    "            pred_score = df1['score'].values[0]\n",
    "            rss_test = rss_test + (pred_score - score)**2\n",
    "    \n",
    "        # 推荐的item总数(TP+FP)\n",
    "        n_total_rec_items = n_total_rec_items + n_rec_items\n",
    "    \n",
    "        # 真实item的总数(TP+FN)\n",
    "        n_total_test_items = n_total_test_items + user_records_test.shape[0]\n",
    "        \n",
    "    # Precision & Recall\n",
    "    precision = n_hits / (1.0 * n_total_rec_items)\n",
    "    recall = n_hits / (1.0 * n_total_test_items)\n",
    "    \n",
    "    # 覆盖度：推荐商品占总需要推荐商品的比例\n",
    "    coverage = len(all_rec_items) / (1.0 * n_items)\n",
    "    \n",
    "    # 打分的均方误差\n",
    "    rmse = np.sqrt(rss_test / df_triplet_test.shape[0])\n",
    "    \n",
    "    print(\"n_rec_items={}时:\\n \\tprecision={}\\n \"\n",
    "          \"\\trecall={}\\n \\tcoverage={}\\n \\trmse={}\".format(n_rec_items, \n",
    "                                                           precision, \n",
    "                                                           recall, \n",
    "                                                           coverage, \n",
    "                                                           rmse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-08T08:23:00.719885Z",
     "start_time": "2020-10-08T08:22:41.754499Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "推荐10个电影时的结果:\n",
      "599 is a new item.\n",
      "711 is a new item.\n",
      "814 is a new item.\n",
      "830 is a new item.\n",
      "852 is a new item.\n",
      "857 is a new item.\n",
      "1156 is a new item.\n",
      "1236 is a new item.\n",
      "1309 is a new item.\n",
      "1310 is a new item.\n",
      "1320 is a new item.\n",
      "1343 is a new item.\n",
      "1348 is a new item.\n",
      "1364 is a new item.\n",
      "1373 is a new item.\n",
      "1457 is a new item.\n",
      "1458 is a new item.\n",
      "1492 is a new item.\n",
      "1493 is a new item.\n",
      "1498 is a new item.\n",
      "1505 is a new item.\n",
      "1520 is a new item.\n",
      "1533 is a new item.\n",
      "1536 is a new item.\n",
      "1543 is a new item.\n",
      "1557 is a new item.\n",
      "1561 is a new item.\n",
      "1562 is a new item.\n",
      "1563 is a new item.\n",
      "1565 is a new item.\n",
      "1582 is a new item.\n",
      "1586 is a new item.\n",
      "n_rec_items=10时:\n",
      " \tprecision=0.07777777777777778\n",
      " \trecall=0.01785\n",
      " \tcoverage=0.08969696969696969\n",
      " \trmse=0.9256115716800145\n",
      "\n",
      "\n",
      "推荐20个电影时的结果\n",
      "599 is a new item.\n",
      "711 is a new item.\n",
      "814 is a new item.\n",
      "830 is a new item.\n",
      "852 is a new item.\n",
      "857 is a new item.\n",
      "1156 is a new item.\n",
      "1236 is a new item.\n",
      "1309 is a new item.\n",
      "1310 is a new item.\n",
      "1320 is a new item.\n",
      "1343 is a new item.\n",
      "1348 is a new item.\n",
      "1364 is a new item.\n",
      "1373 is a new item.\n",
      "1457 is a new item.\n",
      "1458 is a new item.\n",
      "1492 is a new item.\n",
      "1493 is a new item.\n",
      "1498 is a new item.\n",
      "1505 is a new item.\n",
      "1520 is a new item.\n",
      "1533 is a new item.\n",
      "1536 is a new item.\n",
      "1543 is a new item.\n",
      "1557 is a new item.\n",
      "1561 is a new item.\n",
      "1562 is a new item.\n",
      "1563 is a new item.\n",
      "1565 is a new item.\n",
      "1582 is a new item.\n",
      "1586 is a new item.\n",
      "n_rec_items=20时:\n",
      " \tprecision=0.08017429193899782\n",
      " \trecall=0.0368\n",
      " \tcoverage=0.1387878787878788\n",
      " \trmse=0.9256115716800145\n"
     ]
    }
   ],
   "source": [
    "# 分别计算推荐10个电影和20个电影时的precision/recall/coverage/rmse\n",
    "print(\"\\n推荐10个电影时的结果:\")\n",
    "evaluate(total_rec_results, 10)\n",
    "\n",
    "print(\"\\n\\n推荐20个电影时的结果\")\n",
    "evaluate(total_rec_results, 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
